# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
from ..items import QidianItem


class NovelsSpider(scrapy.Spider):
    name = 'novels'
    allowed_domains = ['qidian.com']
    start_urls = ['https://www.qidian.com/all?orderId=&page=1&style=1&pageSize=20&siteid=1&pubflag=0&hiddenField=0']
    count = 0

    def parse(self, response):
        uls = response.xpath('//div[contains(@class, "work-filter") and contains(@class, "type-filter")]/ul')
        print('Here: ', len(uls))
        for ul in uls:
            lis = ul.xpath('.//li')
            for li in lis[1:]:
                type_url = li.xpath('.//a/@href').extract()
                yield Request('https:' + str(type_url[0]), callback=self.type_list)

    def type_list(self, response):
        lis = response.xpath('//ul[contains(@class, "all-img-list") and contains(@class, "cf")]/li')
        for li in lis:
            item = QidianItem()
            book_info = li.xpath('.//div[contains(@class,"book-mid-info")]')

            item['name'] = book_info.xpath('.//h4/a/text()').extract()[0]
            item['author'] = book_info.xpath('.//p[contains(@class,"author")]/a[contains(@class,"name")]/text()').extract()[0]
            item['type'] = book_info.xpath('.//p[contains(@class,"author")]/a')[1].xpath('.//text()').extract()[0]
            item['sub_type'] = book_info.xpath('.//p[contains(@class,"author")]/'
                                                     'a[contains(@class,"go-sub-type")]/text()').extract()[0]
            item['desc'] = book_info.xpath('.//p[contains(@class,"intro")]/text()').extract()[0]
            item['status'] = book_info.xpath('.//p[contains(@class,"author")]/span/text()').extract()[0]

            yield item

        final_li = response.xpath('//ul[contains(@class, "lbf-pagination-item-list")]/li')[-1]
        final_url = final_li.xpath('.//a/@href').extract()[0]
        if final_url != 'javascript:;':
            yield response.follow('https:' + str(final_url), callback=self.type_list)




